{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/zheyuanzhang/anaconda3/envs/FRS/lib/python3.8/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "from RCSYS_utils import *\n",
    "from RCSYS_models import *\n",
    "from utils import *\n",
    "\n",
    "import os\n",
    "os.environ['OPENAI_API_KEY'] = 'YOUR_API_KEY'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Benchmark Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = torch.load('../processed_data/benchmark_all.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the seed for reproducibility\n",
    "np.random.seed(42)\n",
    "\n",
    "# Count the occurrences of each food_id in the edge_index\n",
    "edge_index = data[('user', 'eats', 'food')].edge_index\n",
    "food_ids, counts = edge_index[1].unique(return_counts=True)\n",
    "\n",
    "# Filter out food_ids that have been consumed over 1000 times\n",
    "frequent_food_ids = food_ids[counts > 1000].tolist()\n",
    "mask = torch.tensor([food_id not in frequent_food_ids for food_id in edge_index[1]])\n",
    "\n",
    "# Filter the edge index based on the mask\n",
    "filtered_edge_index = edge_index[:, mask]\n",
    "\n",
    "# Step 1: Filter edge_label_index based on tag sums\n",
    "user_tags = data['user']['tags']\n",
    "food_tags = data['food']['tags']\n",
    "\n",
    "user_mask = user_tags.sum(dim=1) > 5\n",
    "food_mask = food_tags.sum(dim=1) > 5\n",
    "\n",
    "edge_label_index = data[('user', 'eats', 'food')].edge_index\n",
    "user_indices = edge_label_index[0]\n",
    "food_indices = edge_label_index[1]\n",
    "\n",
    "filtered_edges = edge_label_index[:, user_mask[user_indices] & food_mask[food_indices]]\n",
    "\n",
    "# Step 2: Combine user and food prompts into a DataFrame\n",
    "user_ids = filtered_edges[0].numpy()\n",
    "food_ids = filtered_edges[1].numpy()\n",
    "\n",
    "user_prompts = [data['user']['prompt'][i] for i in user_ids]\n",
    "food_prompts = [data['food']['prompt'][i] for i in food_ids]\n",
    "\n",
    "user_tags = data['user']['tags'][user_ids].numpy()\n",
    "food_tags = data['food']['tags'][food_ids].numpy()\n",
    "\n",
    "combined_prompts = [\"{} {}\".format(up, fp) for up, fp in zip(user_prompts, food_prompts)]\n",
    "\n",
    "# Tag names\n",
    "tag_names = [\n",
    "    'low calorie', 'high calorie', 'low protein', 'high protein',\n",
    "    'low carb', 'high carb', 'low sugar', 'high sugar', 'low fiber',\n",
    "    'high fiber', 'low saturated fat', 'high saturated fat',\n",
    "    'low cholesterol', 'high cholesterol', 'low sodium', 'high sodium',\n",
    "    'low calcium', 'high calcium', 'low phosphorus', 'high phosphorus',\n",
    "    'low potassium', 'high potassium', 'low iron', 'high iron',\n",
    "    'low folic acid', 'high folic acid', 'low vitamin c', 'high vitamin c',\n",
    "    'low vitamin d', 'high vitamin d', 'low vitamin b12', 'high vitamin b12'\n",
    "]\n",
    "\n",
    "# Create matching and contradicting columns\n",
    "matching_tags = []\n",
    "contradicting_tags = []\n",
    "\n",
    "for u_tags, f_tags in zip(user_tags, food_tags):\n",
    "    matching = []\n",
    "    contradicting = []\n",
    "    for i in range(0, len(tag_names), 2):\n",
    "        if u_tags[i] == f_tags[i] == 1:\n",
    "            matching.append(tag_names[i])\n",
    "        elif u_tags[i+1] == f_tags[i+1] == 1:\n",
    "            matching.append(tag_names[i+1])\n",
    "        elif u_tags[i] == 1 and f_tags[i+1] == 1:\n",
    "            contradicting.append(tag_names[i+1])\n",
    "        elif u_tags[i+1] == 1 and f_tags[i] == 1:\n",
    "            contradicting.append(tag_names[i])\n",
    "    matching_tags.append(matching)\n",
    "    contradicting_tags.append(contradicting)\n",
    "\n",
    "# Create ground_truth column\n",
    "ground_truth = []\n",
    "for matching, contradicting in zip(matching_tags, contradicting_tags):\n",
    "    if len(matching) > len(contradicting):\n",
    "        ground_truth.append(f\"Yes, because the food is {', '.join(matching)}\")\n",
    "    else:\n",
    "        ground_truth.append(f\"No, because the food is {', '.join(contradicting)}\")\n",
    "\n",
    "# Create DataFrame\n",
    "df = pd.DataFrame({\n",
    "    'user_id': user_ids,\n",
    "    'food_id': food_ids,\n",
    "    'user_tags': list(user_tags),\n",
    "    'food_tags': list(food_tags),\n",
    "    'user_food_combined_prompts': combined_prompts,\n",
    "    'matching_tags': matching_tags,\n",
    "    'contradicting_tags': contradicting_tags,\n",
    "    'ground_truth': ground_truth\n",
    "})\n",
    "\n",
    "# Step 3: Sample 500 examples where matching > contradicting and 500 where contradicting > matching\n",
    "matching_gt_contradict_df = df[df['matching_tags'].apply(len) > df['contradicting_tags'].apply(len)]\n",
    "contradicting_gt_matching_df = df[df['contradicting_tags'].apply(len) > df['matching_tags'].apply(len)]\n",
    "\n",
    "sample_matching = matching_gt_contradict_df.sample(n=100, random_state=42)\n",
    "sample_contradicting = contradicting_gt_matching_df.sample(n=100, random_state=42)\n",
    "\n",
    "# Combine the samples\n",
    "final_sample = pd.concat([sample_matching, sample_contradicting])\n",
    "\n",
    "# Create prompt text columns\n",
    "user_food_liked = []\n",
    "food_considered_healthy = []\n",
    "\n",
    "for index, row in final_sample.iterrows():\n",
    "    # Find 5 other food prompts the user has liked\n",
    "    user_id = row['user_id']\n",
    "    user_food_indices = filtered_edges[1][filtered_edges[0] == user_id].numpy()\n",
    "    other_food_prompts = [data['food']['prompt'][i] for i in user_food_indices if i != row['food_id']]\n",
    "    other_food_prompts = [str(prompt) for prompt in other_food_prompts][:5]\n",
    "\n",
    "    food_health_status = \"healthy\" if \"Yes\" in row['ground_truth'] else \"unhealthy\"\n",
    "    liked_prompt = f\"Keep in mind that We now know that this given food is {food_health_status} to the user! Analyze the reason of this food: {row['user_food_combined_prompts']}. This food is liked by the User {user_id}, User {user_id} also liked the following foods: {'; '.join(other_food_prompts)}\"\n",
    "    \n",
    "    user_food_liked.append(liked_prompt)\n",
    "    \n",
    "    # Find 5 other food prompts considered healthy\n",
    "    healthy_food_indices = [food_ids[idx] for idx, (m, c) in enumerate(zip(matching_tags, contradicting_tags)) if len(m) > len(c)]\n",
    "    other_healthy_food_prompts = [data['food']['prompt'][i] for i in healthy_food_indices if i != row['food_id']]\n",
    "    other_healthy_food_prompts = [str(prompt) for prompt in other_healthy_food_prompts][:5]\n",
    "    healthy_prompt = f\"{row['user_food_combined_prompts']} Other food choices considered healthy to this user include: {'; '.join(other_healthy_food_prompts)}. Please use the information to analyze the given food.\"\n",
    "    \n",
    "    food_considered_healthy.append(healthy_prompt)\n",
    "\n",
    "final_sample['user_food_liked'] = user_food_liked\n",
    "final_sample['food_considered_healthy'] = food_considered_healthy\n",
    "\n",
    "# Create user health tags and new prompt column\n",
    "new_prompt_column = []\n",
    "user_health_tags = []\n",
    "\n",
    "for index, row in final_sample.iterrows():\n",
    "    # Extract user health tags\n",
    "    user_health_tags_text = ', '.join([tag_names[i] for i in range(len(tag_names)) if row['user_tags'][i] == 1])\n",
    "    user_health_tags.append(user_health_tags_text)\n",
    "\n",
    "    # Create user health text\n",
    "    user_health_text = f\"Given user's health condition, please focus on the following nutrient aspect in the analysis: if the food is {user_health_tags_text}\"\n",
    "\n",
    "    # Sample 5 other foods' ground truth\n",
    "    other_foods = final_sample.sample(n=5, random_state=42).reset_index()\n",
    "    other_foods_ground_truth = []\n",
    "    for i, other_row in other_foods.iterrows():\n",
    "        food_health_status = \"healthy\" if \"Yes\" in other_row['ground_truth'] else \"unhealthy\"\n",
    "        other_foods_ground_truth.append(f\"Food {i+1} is considered {food_health_status}, because the food is {other_row['ground_truth'].split(', because the food is ')[-1]}\")\n",
    "\n",
    "    other_foods_text = \"; \".join(other_foods_ground_truth)\n",
    "    food_health_status = \"healthy\" if \"Yes\" in row['ground_truth'] else \"unhealthy\"\n",
    "\n",
    "    # Combine into new prompt\n",
    "    new_prompt = f\"{user_health_text}. And here are 5 additional foods as additional information. {other_foods_text}. Important! Keep in mind that We now know that this given food is {food_health_status} to the user! Please use the information to analyze this food: {row['user_food_combined_prompts']}\"\n",
    "    new_prompt_column.append(new_prompt)\n",
    "\n",
    "final_sample['new_prompt'] = new_prompt_column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Yes, User Node 34469: The user information is as follows: female, age 22, White, household income level (the higher the better): 7, education status: Some college or AA degree. Food Node 92410310: The food description is: Soft drink, cola. This food belongs to the category: Soft drinks. The ingredients in this food are: Beverages, carbonated, cola, regular.. This food is liked by the User 1956, User 1956 also liked the following foods: Food Node 94000100: The food description is: Water, tap. This food belongs to the category: Tap water. The ingredients in this food are: Beverages, water, tap, drinking.; Food Node 57123000: The food description is: Cereal (General Mills Cheerios). This food belongs to the category: Ready-to-eat cereal, lower sugar (=<21.2g/100g). The ingredients in this food are: Cereals ready-to-eat, GENERAL MILLS, CHEERIOS, Vitamin C as ingredient, Cereals, oats, regular and quick, not fortified, dry, Sugars, granulated, Salt, table, iodized, Vitamin B composite in cereals, Folic acid as ingredient, Iron as ingredient, Calcium as ingredient, Vitamin D as ingredient, Fiber, total dietary, as ingredient.; Food Node 11111000: The food description is: Milk, whole. This food belongs to the category: Milk, whole. The ingredients in this food are: Milk, whole, 3.25% milkfat, with added vitamin D.; Food Node 91101010: The food description is: Sugar, white, granulated or lump. This food belongs to the category: Sugars and honey. The ingredients in this food are: Sugars, granulated.; Food Node 92101000: The food description is: Coffee, brewed. This food belongs to the category: Coffee. The ingredients in this food are: Beverages, coffee, brewed, prepared with tap water.'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_sample['user_food_liked'].iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_sample.to_csv('../processed_data/benchmark_reasoning.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Querying"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "\n",
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from transformers import BertTokenizer, BertModel\n",
    "import bert_score\n",
    "from nltk.translate.bleu_score import sentence_bleu\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "client = OpenAI()\n",
    "\n",
    "base_text = \"\"\"\n",
    "    Act as a nutritionist, your task is to use your knowledge to judge if the given food should be considered a healthy option to the user given their profiles. \n",
    "    Important Note: You must strictly provide your answer in the following format without further explanations or other words:  <Yes or No>, because the food is: <high or low> in <nutrients>, <choose between high or low> in <nutrients>,…. (E.g. Yes, becuase the food is low in calories, low in sodium and high in protein).\n",
    "    Here is the input: \n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def query_GPT(final_sample, context_column, output_path):\n",
    "    prompted_text_list = []\n",
    "    for i, row in final_sample.iterrows():\n",
    "        prompted_text = base_text + row[context_column]\n",
    "        prompted_text_list.append(prompted_text)\n",
    "\n",
    "    def query(prompt):\n",
    "        response = client.chat.completions.create(\n",
    "          model=\"gpt-3.5-turbo\",\n",
    "          messages=[\n",
    "            {\"role\": \"system\", \"content\": \"Act as a nutritionist. Ananlyze if a given food is healthy to a user and why.\"},\n",
    "            {\"role\": \"user\", \"content\": prompt}\n",
    "          ]\n",
    "        )\n",
    "        return response.choices[0].message.content\n",
    "\n",
    "    \n",
    "    GPT_reasoning = []\n",
    "    for prompt in tqdm(prompted_text_list):\n",
    "        reasoning = query(prompt)\n",
    "        time.sleep(10)\n",
    "        GPT_reasoning.append(reasoning)\n",
    "\n",
    "    # Create a DataFrame\n",
    "    gpt_results_df = pd.DataFrame({'GPT_results': GPT_reasoning})\n",
    "\n",
    "    # Save to CSV\n",
    "    gpt_results_csv_path = '../processed_data/reasoning/' + output_path\n",
    "    gpt_results_df.to_csv(gpt_results_csv_path, index=False)\n",
    "\n",
    "    return GPT_reasoning\n",
    "\n",
    "\n",
    "def evaluate(GPT_reasoning, df):\n",
    "    # Sample ground truth and GPT-generated results\n",
    "    ground_truths = df['ground_truth'].tolist()  # List of ground truth texts\n",
    "    gpt_results = GPT_reasoning\n",
    "\n",
    "    # Calculate BERT scores\n",
    "    P, R, F1 = bert_score.score(gpt_results, ground_truths, lang=\"en\", model_type=\"bert-base-uncased\")\n",
    "    bert_scores = F1.numpy()\n",
    "    bert_score_mean = np.mean(bert_scores)\n",
    "    bert_score_std = np.std(bert_scores)\n",
    "\n",
    "    # Calculate BLEU scores\n",
    "    gpt_scores = [sentence_bleu([gt.split()], gpt.split()) for gt, gpt in zip(ground_truths, gpt_results)]\n",
    "    bleu_score_mean = np.mean(gpt_scores)\n",
    "    bleu_score_std = np.std(gpt_scores)\n",
    "\n",
    "    # Create DataFrame for results\n",
    "    result_df = pd.DataFrame({\n",
    "        'GPT_results': gpt_results,\n",
    "        'BERT_score': bert_scores,\n",
    "        # 'BLEU_score': gpt_scores\n",
    "    })\n",
    "\n",
    "    # Summary\n",
    "    summary = {\n",
    "        'BERT_score_mean': bert_score_mean,\n",
    "        'BERT_score_std': bert_score_std,\n",
    "        'BLEU_score_mean': bleu_score_mean,\n",
    "        'BLEU_score_std': bleu_score_std\n",
    "    }\n",
    "\n",
    "    # Output results\n",
    "    print(summary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# raw_reasoning = pd.read_csv('../processed_data/reasoning/raw_GPT.csv')['GPT_results'].tolist()\n",
    "# xrec_reasoning = pd.read_csv('../processed_data/reasoning/XRec_GPT.csv')['GPT_results'].tolist()\n",
    "# llm2er_reasoning = pd.read_csv('../processed_data/reasoning/LLM2ER_GPT.csv')['GPT_results'].tolist()\n",
    "# our_reasoning = pd.read_csv('../processed_data/reasoning/Ours_GPT.csv')['GPT_results'].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'BERT_score_mean': 0.6903412, 'BERT_score_std': 0.04729, 'BLEU_score_mean': 0.24047353478215364, 'BLEU_score_std': 0.0883334715206406}\n"
     ]
    }
   ],
   "source": [
    "# Raw \n",
    "raw_reasoning = query_GPT(final_sample, context_column='user_food_combined_prompts', output_path='raw_GPT.csv')\n",
    "evaluate(raw_reasoning, final_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'BERT_score_mean': 0.6992225, 'BERT_score_std': 0.03730052, 'BLEU_score_mean': 0.25539204386224, 'BLEU_score_std': 0.0773054282350995}\n"
     ]
    }
   ],
   "source": [
    "# Xrec\n",
    "xrec_reasoning = query_GPT(final_sample, context_column='food_considered_healthy', output_path='XRec_GPT.csv')\n",
    "evaluate(xrec_reasoning, final_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'BERT_score_mean': 0.6948765, 'BERT_score_std': 0.046850063, 'BLEU_score_mean': 0.24030639585726427, 'BLEU_score_std': 0.11347167381448237}\n"
     ]
    }
   ],
   "source": [
    "# LLM2ER\n",
    "llm2er_reasoning = query_GPT(final_sample, context_column='user_food_liked', output_path='LLM2ER_GPT.csv')\n",
    "evaluate(llm2er_reasoning, final_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200/200 [35:19<00:00, 10.60s/it]\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `beta` will be renamed internally to `bias`. Please use a different name to suppress this warning.\n",
      "A parameter name that contains `gamma` will be renamed internally to `weight`. Please use a different name to suppress this warning.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'BERT_score_mean': 0.7494171, 'BERT_score_std': 0.075974554, 'BLEU_score_mean': 0.2868334327549, 'BLEU_score_std': 0.12916104869960582}\n"
     ]
    }
   ],
   "source": [
    "our_reasoning = query_GPT(final_sample, context_column='new_prompt', output_path='Ours_GPT.csv')\n",
    "evaluate(our_reasoning, final_sample)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "FRS",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
